{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c557dcdf",
   "metadata": {},
   "source": [
    "# Extending FLiOS Scenarios\n",
    "\n",
    "This notebook demonstrates how to download and preprocess further benchmarking datasets and its associated machine learning models for the extension of the FLiOS application."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba1186b8",
   "metadata": {},
   "source": [
    "## 1. Loading Data\n",
    "\n",
    "In the following, functions for loading datasets are defined and executes. The data are loaded with the keras package and saved to the local storage using the .csv file format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5be152aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tensorflow import keras\n",
    "import coremltools as ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686c227b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist():\n",
    "    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "    df_train = pd.DataFrame(\n",
    "        x_train.reshape(x_train.shape[0], x_train.shape[1] * x_train.shape[2])\n",
    "    )\n",
    "    df_train[\"target\"] = y_train\n",
    "    df_train.to_csv(\"./mnist_train.txt\", index=False, header=False)\n",
    "    df_test = pd.DataFrame(\n",
    "        x_test.reshape(x_test.shape[0], x_test.shape[1] * x_test.shape[2])\n",
    "    )\n",
    "    df_test[\"target\"] = y_test\n",
    "    df_test.to_csv(\"./mnist_test.txt\", index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4fa8ecc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cifar():\n",
    "    (x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\n",
    "    df_train = pd.DataFrame(\n",
    "        x_train.reshape(\n",
    "            x_train.shape[0], x_train.shape[1] * x_train.shape[2] * x_train.shape[3]\n",
    "        )\n",
    "    )\n",
    "    df_train[\"target\"] = y_train\n",
    "    df_train.to_csv(\"./cifar_train.txt\", index=False, header=False)\n",
    "    df_test = pd.DataFrame(\n",
    "        x_test.reshape(\n",
    "            x_test.shape[0], x_test.shape[1] * x_test.shape[2] * x_test.shape[3]\n",
    "        )\n",
    "    )\n",
    "    df_test[\"target\"] = y_test\n",
    "    df_test.to_csv(\"./cifar_test.txt\", index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f222f213",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_boston():\n",
    "    (x_train, y_train), (x_test, y_test) = keras.datasets.boston_housing.load_data()\n",
    "    df_train = pd.DataFrame(x_train.reshape(x_train.shape[0], x_train.shape[1]))\n",
    "    df_train[\"target\"] = y_train\n",
    "    df_train.to_csv(\"./boston_train.txt\", index=False, header=False)\n",
    "    df_test = pd.DataFrame(x_test.reshape(x_test.shape[0], x_test.shape[1]))\n",
    "    df_test[\"target\"] = y_test\n",
    "    df_test.to_csv(\"./boston_test.txt\", index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16564122",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_mnist()\n",
    "load_cifar()\n",
    "load_boston()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b28faf0",
   "metadata": {},
   "source": [
    "## 2. Compressing\n",
    "\n",
    "The datasets are compressed into Apple's compression format (lzfse) and located in the supposed folder of the iOS project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b374140e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!lzfse -encode -i mnist_train.txt -o FLiOS/Scenarios/MNIST/MNIST_train.csv.lzfse\n",
    "!lzfse -encode -i mnist_test.txt -o FLiOS/Scenarios/MNIST/MNIST_test.csv.lzfse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec8ec27",
   "metadata": {},
   "outputs": [],
   "source": [
    "!lzfse -encode -i cifar_train.txt -o FLiOS/Scenarios/CIFAR/CIFAR_train.csv.lzfse\n",
    "!lzfse -encode -i cifar_test.txt -o FLiOS/Scenarios/CIFAR/CIFAR_test.csv.lzfse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c306ca04",
   "metadata": {},
   "outputs": [],
   "source": [
    "!lzfse -encode -i boston_train.txt -o FLiOS/Scenarios/BOSTON/BOSTON_train.csv.lzfse\n",
    "!lzfse -encode -i boston_test.txt -o FLiOS/Scenarios/BOSTON/BOSTON_test.csv.lzfse"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbbc5efc",
   "metadata": {},
   "source": [
    "## 3. Models\n",
    "\n",
    "Define the keras models and and transform them into a CoreML model using coremltools."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "398f4550",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mnist_model():\n",
    "    # ...\n",
    "    return \"model\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dac2e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cifar_model():\n",
    "    # Create model\n",
    "    model = keras.models.Sequential()\n",
    "\n",
    "    # Add layers\n",
    "    model.add(\n",
    "        keras.layers.Conv2D(\n",
    "            32, kernel_size=(3, 3), activation=\"relu\", input_shape=(32, 32, 3)\n",
    "        )\n",
    "    )\n",
    "    model.add(keras.layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"))\n",
    "    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(keras.layers.Dropout(0.25))\n",
    "\n",
    "    # Layer\n",
    "    model.add(keras.layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\"))\n",
    "    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(keras.layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\"))\n",
    "    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(keras.layers.Dropout(0.25))\n",
    "\n",
    "    # Layer\n",
    "    model.add(keras.layers.Flatten())\n",
    "    model.add(keras.layers.Dense(1024, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dropout(0.5))\n",
    "    model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fefc1fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_boston_model():\n",
    "    model = keras.models.Sequential()\n",
    "    model.add(keras.layers.Dense(100, input_dim=13, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(100, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(1, activation=\"linear\"))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f1c7d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_mnist = get_mnist_model()\n",
    "# model_cifar = get_cifar_model()\n",
    "# model_boston = get_boston_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d212252d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_mnist = ct.convert(model_mnist)\n",
    "# model_cifar = ct.convert(model_cifar)\n",
    "# model_boston = ct.convert(model_boston)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f2c53e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_mnistsave('./FLiOS/Scenarios/MNIST/MNIST_model.mlmodel')\n",
    "# model_cifar.save('./FLiOS/Scenarios/CIFAR/CIFAR_model.mlmodel')\n",
    "# model_boston.save('./FLiOS/Scenarios/BOSTON/BOSTON_model.mlmodel')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
